# Updated by Wei on 2014/03/07 afternoon at school
from struct import *
import sys
import os

# This script is used for generating the aux file for navigating the posting binary file
print "aux file generation"

def auxFileGeneration():
    print "auxFileGeneration() begins..."
    # for pangolin:
    # inputFileName = "/data/obukai/workspace_USE_SINCE_20140217Night/web-search-engine-wei-2014-March/polyIRIndexer/selectedDocumentPostingValuesInfo20140215Afternoon_WHOLE_DOC.binary"
    # inputFileName = "/data/obukai/workspace_USE_SINCE_20140217Night/web-search-engine-wei-2014-March/polyIRIndexer/selectedDocumentPostingValuesInfo20140306Afternoon_PART0_DOC.binary"
    # inputFileName = "/data/obukai/workspace_USE_SINCE_20140217Night/web-search-engine-wei-2014-March/polyIRIndexer/LEAVE_selectedDocumentPostingValuesInfo20140306Afternoon_PART1_DOC.binary"
    inputFileName = "/data/obukai/workspace_USE_SINCE_20140217Night/web-search-engine-wei-2014-March/polyIRIndexer/LEAVE_selectedDocumentPostingValuesInfo20140310Afternoon_PART0_1_DOC.binary"
    # for dodo:
    # inputFileName = "/home/diaosi/workspace/web-search-engine-wei-2014-March/polyIRIndexer/selectedDocumentPostingValuesInfo20140306Afternoon_PART3_DOC.binary"
    # inputFileName = "/home/diaosi/workspace/web-search-engine-wei-2014-March/polyIRIndexer/LEAVE_selectedDocumentPostingValuesInfo20140306Afternoon_PART2_DOC.binary"
    inputFileHandler = open(inputFileName,"rb")
    statinfo = os.stat(inputFileName)
    fileSize = statinfo.st_size
    print "file size:",fileSize

    numOfBytesRead = 0
    numOfDocsProcessed = 0
    # numOfDocsProcessed = 6000000

    
    while numOfBytesRead <= fileSize:
        if numOfDocsProcessed % 100000 == 0:
            print numOfDocsProcessed,numOfBytesRead
        # print "numOfBytesRead:",numOfBytesRead
        byteString = inputFileHandler.read(4 + 4)
        (docID,numOfPostings) = unpack( "2I", byteString)
        # print "docID:",str(docID),str(numOfPostings)
        
        numOfBytesRead += 4
        numOfBytesRead += 4
        numOfBytesRead += numOfPostings * 4 * 5
        
        inputFileHandler.seek(numOfBytesRead)
        
        '''
        for index in range(0,numOfPostings):
            byteString = inputFileHandler.read(4 + 4 + 4)
            (termID,staticProbability,dynamicProbability) = unpack( "1I2f", byteString)
            # print index,termID,staticProbability,dynamicProbability
        '''
        numOfDocsProcessed += 1
    
    print numOfDocsProcessed,numOfBytesRead   
    inputFileHandler.close()
    print "Overall:"
    print "inputFileName:",inputFileName
    
    inputFileHandler.close()
    print "auxFileGeneration() ends."

def verify():
    print "verify() begins..."
    # for pangolin:
    # inputFileName = "/data/obukai/workspace_USE_SINCE_20140217Night/web-search-engine-wei-2014-March/polyIRIndexer/selectedDocumentPostingValuesInfo20140306Afternoon_PART0_DOC.binary"
    inputFileName = "/data/obukai/workspace_USE_SINCE_20140217Night/web-search-engine-wei-2014-March/polyIRIndexer/LEAVE_selectedDocumentPostingValuesInfo20140306Afternoon_PART1_DOC.binary"
    # for dodo:
    # inputFileName = "/home/diaosi/workspace/web-search-engine-wei-2014-March/polyIRIndexer/selectedDocumentPostingValuesInfo20140306Afternoon_PART4_DOC.binary"
    # inputFileName = "/home/diaosi/workspace/web-search-engine-wei-2014-March/polyIRIndexer/LEAVE_selectedDocumentPostingValuesInfo20140306Afternoon_PART3_DOC.binary"
    # inputFileName = "/home/diaosi/workspace/web-search-engine-wei-2014-March/polyIRIndexer/selectedDocumentPostingValuesInfo20140306Afternoon_PART4_DOC.binary"
    inputFileHandler = open(inputFileName,"rb")
    statinfo = os.stat(inputFileName)
    fileSize = statinfo.st_size
    print "file size:",fileSize

    numOfBytesRead = 13040502620
    numOfDocsProcessed = 5900000
    # numOfBytesRead = 27376927280
    # numOfDocsProcessed = 18400000
    inputFileHandler.seek(numOfBytesRead)
    
    while numOfBytesRead < fileSize:
        
        # print "numOfBytesRead:",numOfBytesRead
        byteString = inputFileHandler.read(4 + 4)
        (docID,numOfPostings) = unpack( "2I", byteString)
        print "docID:",str(docID),str(numOfPostings)
        
        numOfBytesRead += 4
        numOfBytesRead += 4

        if docID != 25128547:
            for index in range(0,numOfPostings):
                byteString = inputFileHandler.read(4 + 4 + 4 + 4 + 4)
                (termID,staticProbability,dynamicProbability,combineProbability,partialBM25) = unpack( "1I4f", byteString)
                print index,termID,staticProbability,dynamicProbability,combineProbability,partialBM25
            exit(1);
        numOfBytesRead += numOfPostings * 4 * 5
        inputFileHandler.seek(numOfBytesRead)        
        numOfDocsProcessed += 1
    
    print numOfDocsProcessed,numOfBytesRead   
    inputFileHandler.close()
    print "Overall:"
    print "inputFileName:",inputFileName
    
    inputFileHandler.close()
    print "verify() ends."

auxFileGeneration()
# verify()
print "End"

